import sys, os, re, socket, time, thread, random

dbggrp = [
           False, # get_gs3_protocol_data raw data
           False, # get_gs3_protocol_data decoded data
           False, # get_gsweb_data
           False  # main loop
         ]
###############################################################################
# Gamespy3 Protocol Response Structure
#
# ========== ORDERING AND TOTAL NUMBER OF DATAGRAMS ==========
# \x00\x10 0@splitnum\x00 is the actual datagram header - I don't know what it means and it never changes.
# \x00\x10 0@splitnum\x00\x??
#                          ^
#                           \_ the MSB (0x80) indicates this is the last datagram
#                              the LSB indicates the datagram number
#
# ========== SECTION NUMBERING ==========
# \x00\x10 0@splitnum\x00\x??\x00hostname\x00...
#                              ^
#                               \_ indicates start of "section 0" (server section - hostname, gamename, gamever, etc)
# ...bf2_novehicles\x001\x00\x00\x01player_\x00\x00DuNE...
#                             ^   ^
#                             |    \_ indicates start of "section 1" (player section - player_, score_, ping_, etc)
#                             the extra \x00 indicates the end of a section (end of "section 0" in this case)
# ...\x00\x00\x00\x02team_t\x00\x00MEC\x00...
#              ^   ^             ^
#              |    \             \_ "next element number" (see below)
#              |     \_ indicates start of "section 2" (team section - team_t, score_t)
#              the extra \x00 indicates the end of a section (end of "section 1" in this case)
#
# ========== SPANNING DATAGRAMS ==========
# Fields within "section 1" and "section 2" are followed by a byte indicating which element number is coming next.
# In the case of spanned datagrams, the subsequent datagram will begin with something like:
# \x00\x10 0@splitnum\x00\x??\x01score_\x00\x??
#                          ^   ^   ^         ^
#       "datagram order" _/    |   |          \_ "next element number" (zero based)
#                     "section#"  "field name"
# It will be necessary to overwrite any partial data element previously stored in a field with
# the full data element from the next datagram.
#
# Section 0 contains only one data element for each field and therefore does not need a "next element number" after each field name.
# Sections 1 and 2 contain multiple data elements for each field and therefore do need a "next element number" after each field name.
# Underscore as the last character of the field name indicates that this is an array field.  For example: "player_"
###############################################################################

###############################################################################
# servers = [
#             {
#               ip: xxx.xxx.xxx.xxx,
#               port: xxxxx,
#               hostname: "Server Name String",
#               mapname: "Map Name String",
#               numplayers: xx,
#               ping: xxx,
#               blowoutratio: x.xx,
#               avgrank: xx.x,
#               stdevrank: x.x,
#               players: [
#                          {  player_: "Player Name String",
#                             pid_: xxxxxxx,
#                             team_: x, (1 or 2.  Team Name String indicated by bf2_team1 and bf2_teams)
#                             score_: xx, (this is current score on server)
#                             rank: xx,
#                             scor: xxxxx (this is global score)
#                          },
#                          ...
#                        ]
#               teams:   [
#                          {  team_t: "Team Name String",
#                             score_t: xx,
#                             avgTeamScore: xx
#                          },
#                          ...
#                        ]
#             },
#             ...
#           ]
###############################################################################

########################################################################################################################################

def output(msg, outfile, stdoutEnable):
    if stdoutEnable == True:
        print msg
    if outfile != None:
        print >>outfile, msg

def debug(msg, enabled):
    if enabled : print msg
    
########################################################################################################################################

def gs3_protocol_worker_thread(threadName, server): #download server and player data from this server

    # global thread communication variables and thier associated mutual exclusion locks
    global numThreads, numThreadsLock, serversLock

    output(server["ip"] + ":" + server["port"], outfile, True)
    debug("----------------", dbggrp[1])
    debug("beginning of " + server["ip"] + ":" + server["port"], dbggrp[1])
    debug("----------------", dbggrp[1])

    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
    s.settimeout(1)
    s.connect((server["ip"], int(server["port"])))
    s.send("\xfe\xfd\x00\x10\x20\x30\x40\xff\xff\xff\x01")

    listOfDatagramTuples =[]
    isCompleteSetOfDatagrams = False
    while not isCompleteSetOfDatagrams:
        try:
            datagram = s.recv(2048)
            datagramNum = ord(datagram[14]) & ord("\x7f") # mask off the MSB
            lastDatagram = bool(ord(datagram[14]) & ord("\x80")) # MSB indicates lastDatagram

            # check if we have a complete, ordered set of datagrams
            listOfDatagramTuples.append((datagramNum, lastDatagram, datagram))
            listOfDatagramTuples.sort(key=lambda x: x[0])
            datagramsAreConsecutive = True
            for i in xrange(len(listOfDatagramTuples)):
                if listOfDatagramTuples[i][0] != i:
                    datagramsAreConsecutive = False
            if datagramsAreConsecutive == True and listOfDatagramTuples[-1][1] == True: #if lastDatagram is present
                isCompleteSetOfDatagrams = True

        except (socket.timeout, socket.error): #we gave you enough time, we're moving on now
            output(server["ip"] + ":" + server["port"] + " socket timed out or error", outfile, True)
            # this is rediculous, but there is no good way to delete an element from a list you are iterating over in Python
            # because it messes up the indexing when the length of the list changes
            # so just flag the server for deletion outside the loop
            serversLock.acquire()
            server["hasErrors"] = True
            serversLock.release()
            break

    debug("----------------", dbggrp[1])
    debug("listOfDatagramTuples : " + `listOfDatagramTuples`, dbggrp[1])
    debug("----------------", dbggrp[1])

    if isCompleteSetOfDatagrams:
        for datagramTuple in listOfDatagramTuples:
            datagramNum, lastDatagram, data = datagramTuple
            try:
                # parse the datagram data into a python data structure
                debug("----------------", dbggrp[1])
                debug("datagramNum : " + `datagramNum`, dbggrp[1])
                debug("lastDatagram : " + `lastDatagram`, dbggrp[1])
                debug(`data`, dbggrp[1])
                debug("----------------", dbggrp[1])
                data = data[15:] # keep throwing away the bytes we've finished looking at

                while True: # process sections until we run out of data and get an IndexError
                    sectionNum = ord(data[0])
                    data = data[1:]
                    debug("----------------", dbggrp[1])
                    debug("sectionNum : " + `sectionNum`, dbggrp[1])
                    debug("----------------", dbggrp[1])

                    while ord(data[0]) != 0: # if we're expecting a fieldName and get a "\x00" instead, that signals the start of a new section
                        fieldName, sep, data = data.partition("\x00")
                        debug("fieldName : " + fieldName, dbggrp[1])
                        debug("data      : " + `data`, dbggrp[0])

                        if sectionNum == 0 : # process scalars
                            serversLock.acquire()
                            server[fieldName], sep, data = data.partition("\x00")
                            serversLock.release()
                            debug("fieldValue : " + server[fieldName], dbggrp[1])
                            debug("data       : " + `data`, dbggrp[0])
                            debug("----------------", dbggrp[1])

                        else : # process arrays
                            elementNum = ord(data[0])
                            data = data[1:]
                            debug("----------------", dbggrp[1])
                            debug("elementNum : " + `elementNum`, dbggrp[1])
                            debug("----------------", dbggrp[1])

                            #Check to see if we are creating a new field, or appending an existing one (due to datagram spanning)
                            if elementNum == 0:
                                serversLock.acquire()
                                server[fieldName] = []
                                serversLock.release()
                            else:
                                serversLock.acquire()
                                server[fieldName] = server[fieldName][:elementNum]
                                serversLock.release()

                            while ord(data[0]) != 0: # if we're expecting an elementValue and get a "\x00" instead, that signals the start of a new fieldName
                                elementValue, sep, data = data.partition("\x00")
                                debug("elementValue : " + elementValue, dbggrp[1])
                                debug("data         : " + `data`, dbggrp[0])
                                debug("----------------", dbggrp[1])

                                serversLock.acquire()
                                server[fieldName].append(elementValue)
                                serversLock.release()

                            data = data[1:] # need to consume the loop termination byte

                    data = data[1:] # need to consume the loop termination byte

            except IndexError:
                debug("----------------", dbggrp[1])
                debug("end of datagramNum : " + `datagramNum`, dbggrp[1])
                debug("----------------", dbggrp[1])

        debug("----------------", dbggrp[1])
        debug("end of " + server["ip"] + ":" + server["port"], dbggrp[1])
        debug("----------------", dbggrp[1])

    numThreadsLock.acquire()
    numThreads-=1
    numThreadsLock.release()
########################################################################################################################################

def get_gs3_protocol_data(servers): #download server and player data from this server

    # global thread communication variables and thier associated mutual exclusion locks
    global numThreads, numThreadsLock, serversLock

    numThreadsLock = thread.allocate_lock()
    serversLock = thread.allocate_lock()
    numThreads = 0

    for server in servers :
        while numThreads > 50: pass
        numThreadsLock.acquire()
        numThreads+=1
        numThreadsLock.release()
        thread.start_new_thread(gs3_protocol_worker_thread, ("Thread"+server["ip"], server))
    while numThreads != 0: pass

    # delete all the servers that hasErrors - http://www.daniweb.com/forums/thread73944.html
    for server in servers:
        if server.has_key("hasErrors"):
            output("Deleting " + server["ip"] + ":" + server["port"] + " because it had an error.", outfile, True)
    servers_copy = [ server for server in servers if not server.has_key("hasErrors") ]
    del servers[:]
    servers.extend(servers_copy)
########################################################################################################################################

def convertLists2Dict(servers):

    for server in servers:
        playerFieldlist = []
        teamFieldlist = []

        #Find player and team lists
        for fieldName in server:
            if fieldName[-1] == "_": #Found Player List
                playerFieldlist.append(fieldName)
            elif fieldName[-2:] == "_t": #Found Team List
                teamFieldlist.append(fieldName)

        server["players"] = []
        server["teams"] = []
        # Move Player Values over to new dictionary data structure
        # just pick the first field from the playerFieldlist to figure the length, since all the player fields must have the same length data lists
        # do not trust the "numplayers" field to match the number of elements in the player fields, they don't always agree (maybe a player leaves right as the stats are read?)
        for i in xrange(len(server[playerFieldlist[0]])):
            server["players"].append({})
            for fieldName in playerFieldlist:
                server["players"][i][fieldName] = server[fieldName][i]

        # Move Team Values over to new dictionary data structure
        for i in xrange(len(server[teamFieldlist[0]])):
            server["teams"].append({})
            for fieldName in teamFieldlist:
                server["teams"][i][fieldName] = server[fieldName][i]

        #Delete player and team lists from server
        for fieldName in playerFieldlist:
            del server[fieldName]
        for fieldName in teamFieldlist:
            del server[fieldName]

        # delete all the players with bots with (pid_==0)- http://www.daniweb.com/forums/thread73944.html
        for player in server["players"]:
            if player["pid_"] == "0":
                output("Deleting player " + player["player_"] + ":" + player["pid_"] + " because they have a PID of 0.", outfile, True)
        players_copy = [ player for player in server["players"] if not player["pid_"]=="0" ]
        del server["players"][:]
        server["players"].extend(players_copy)
########################################################################################################################################

def gsweb_player_worker_thread(threadName, player, ip): #look up rank and other player specific stats from the web

    # global thread communication variables and thier associated mutual exclusion locks
    global numThreads, numThreadsLock, serversLock
    
    gamespy_getrequest = "GET /ASP/getplayerinfo.aspx?pid=%s&info=per*,cmb*,twsc,cpcp,cacp,dfcp,kila,heal,rviv,rsup,rpar,tgte,dkas,dsab,cdsc,rank,cmsc,kick,kill,deth,suic,ospm,klpm,klpr,dtpr,bksk,wdsk,bbrs,tcdr,ban,dtpm,lbtl,osaa,vrk,tsql,tsqm,tlwf,mvks,vmks,mvn*,vmr*,fkit,fmap,fveh,fwea,wtm-,wkl-,wdt-,wac-,wkd-,vtm-,vkl-,vdt-,vkd-,vkr-,atm-,awn-,alo-,abr-,ktm-,kkl-,kdt-,kkd- HTTP/1.1\r\n\
Host: BF2Web.gamespy.com\r\n\
User-Agent: GameSpyHTTP/1.0\r\n\
Connection: close\r\n\r\n"

    t1 = time.time()
    del player["hasErrors"]
    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
    s.settimeout(1)
    try:
        s.connect((ip, 80))
        s.send(gamespy_getrequest % player["pid_"])
        response = ""
        while True:
            data = s.recv(4096)
            if data == "":
                break
            else:
                response += data
    except (socket.timeout, socket.error):
        output("Player " + player["pid_"] + " socket timed out or error", outfile, True)
        player["hasErrors"] = True
        #time.sleep(random.uniform(0, 10)) #congestion control
        numThreadsLock.acquire()
        numThreads-=1
        numThreadsLock.release()
        sys.exit(0) #kill this thread
    (keys, values) = response.split("\n")[16:18]

    debug("----------------", dbggrp[2])
    debug(threadName+": gamespy_getrequest = " + gamespy_getrequest % player["pid_"], dbggrp[2])
    debug(threadName+": keys = " + keys, dbggrp[2])
    debug(threadName+": values = " + values, dbggrp[2])

    keys = keys.split()[1:]
    values = values.split()[1:]
    if len(keys) != 236:
        player["hasErrors"] = True
    else:
        serversLock.acquire()
        player.update(dict(zip(keys, values)))
        serversLock.release()
    
        for key in sorted(player.keys()):
            debug(threadName+": " + `key` + ": " + `player[key]`, dbggrp[2])
        debug("----------------", dbggrp[2])

    output("%.2f seconds elapsed for %s" % (time.time() - t1, threadName), outfile, True)
    numThreadsLock.acquire()
    numThreads-=1
    numThreadsLock.release()
########################################################################################################################################

def get_gsweb_data(servers):

    global numThreads, numThreadsLock, serversLock

    numThreadsLock = thread.allocate_lock()
    serversLock = thread.allocate_lock()
    numThreads = 0
    ip = socket.gethostbyname("bf2web.gamespy.com")

    #initialize "hasErrors" value so each player gets looked up initially
    for server in servers:
        for player in server["players"] :
            player["hasErrors"] = True

    errorFree = False
    while not errorFree:
        for server in servers:
            for player in server["players"] :
                if player.has_key("hasErrors"):
                    while numThreads > 15: pass
                    numThreadsLock.acquire()
                    numThreads+=1
                    numThreadsLock.release()
                    thread.start_new_thread(gsweb_player_worker_thread, ("Thread"+player["pid_"], player, ip))
        while numThreads != 0: pass

        errorFree = True
        for server in servers:
            for player in server["players"]:
                if player.has_key("hasErrors"):
                    errorFree = False
        if not errorFree:
            output("Retrying players with errors...", outfile, True)
########################################################################################################################################

def augmentData(servers):

    #data fields explained.  ospm looks interesting - Overall score/minute
    #http://bf2.fun-o-matic.org/index.php/Getplayerinfo_columns

    for server in servers:
        ospmSum = tm1score = tm2score = tm1players = tm2players = 0
        server["maxOSPM"] = 0
    
        try:
            for player in server["players"]:
                if float(player["ospm"]) > float(server["maxOSPM"]):
                    server["maxOSPM"] = float(player["ospm"])
                ospmSum += float(player["ospm"])
                if player["team_"] == "1":
                    tm1score += int(player["scor"])
                    tm1players += 1
                elif player["team_"] == "2":
                    tm2score += int(player["scor"])
                    tm2players += 1
        except KeyError:
            output("server = " + `server`, outfile, True)
            output("player = " + `player`, outfile, True)
            raise

        try:                      server["teams"][0]["avgTeamScore"] = tm1score / tm1players
        except ZeroDivisionError: server["teams"][0]["avgTeamScore"] = 0
        try:                      server["teams"][1]["avgTeamScore"] = tm2score / tm2players
        except ZeroDivisionError: server["teams"][1]["avgTeamScore"] = 0
        try:                      server["avgGlobalScore"] = (tm1score + tm2score) / (tm1players + tm2players)
        except ZeroDivisionError: server["avgGlobalScore"] = 0
        try:                      server["avgOSPM"] = ospmSum / (tm1players + tm2players)
        except ZeroDivisionError: server["avgOSPM"] = 0
########################################################################################################################################

def displayResults(servers):

    #http://wiki.python.org/moin/HowTo/Sorting

    servers.sort(key=lambda x: float(x["avgOSPM"]))
    for server in servers:
        output("-------BEGIN SERVER DATA-----------", outfile, False)
        for key in sorted(server.keys()):
            if key == "players" or key == "teams":
                output("    -----"+key+"------", outfile, False)
                for lst in server[key]:
                    for key2 in sorted(lst.keys()):
                        output("    " + key2 + ": " + `lst[key2]`, outfile, False)
                    output("    ------------------", outfile, False)
            else:
                output(key + ": " + `server[key]`, outfile, False)
########################################################################################################################################

#TODO build GUI with PyGTK
#TODO add "vehicles only" filter and make these adjustable from outside the program...
#     better upfront filters will cut runtime substantially
#TODO gslist opens a TCP connection with master.gamespy.com:28900 to retrieve an encrypted list of servers
#     "country" information is obtained using the 3rd party program GeoIP
#     other filters are sent to master.gamespy.com to limit the length of the server list that is returned
#     The conversation with master.gamespy.com:28900 looks like this:
#     \basic\\secure\AKDKPN.
#     \gamename\gamespy2\gamever\20603020\enctype\1\validate\uC2a+r4A\final\\list\cmp\gamename\battlefield2\where\(numplayers > 0)
#TODO score_t is always zero?

if len(sys.argv) == 2:
    outfile = open(sys.argv[1], "w")
else:
    outfile = None

output("Finding the suckiest Battelfield 2 server to dominate on...", outfile, True)
ttotal = time.time()
output(time.strftime("%c"), outfile, True)

#get the list of servers from gamespy
output("Getting list of servers from master.gamespy.com server...", outfile, True)
f = os.popen("gslist.exe -n battlefield2 -f \"(numplayers > 6) AND (gamevariant='bf2') AND (gamever='1.1.2965-797.0') AND (country='US')\"", "r")

servers = []
for line in f:
    #use a regular expression to match and remember server ip and port
    p = re.search(r"(?P<ip>\d+.\d+.\d+.\d+)\s+(?P<port>\d+)", line)
    if p :
       servers.append({"ip": p.group("ip"), \
                       "port": p.group("port")})

output(`len(servers)` + " servers found.", outfile, True)

#servers = servers[:25]

output("Processing gs3 protocol data for each server...", outfile, True)
t1 = time.time()
get_gs3_protocol_data(servers)
convertLists2Dict(servers)
output("%.2f seconds to get and process gs3_protocol_data() for all servers." % (time.time() - t1), outfile, True)

output("Processing gsweb protocol data for each player...", outfile, True)
t1 = time.time()
get_gsweb_data(servers)
totalPlayers = 0
for server in servers:
    totalPlayers += len(server["players"])
output("Average througput = %f players per second on %d total players." % (totalPlayers / (time.time() - t1), totalPlayers), outfile, True)
output("%.2f seconds to get and process get_gsweb_data() for all players." % (time.time() - t1), outfile, True)

output("Augmenting data with calculated values...", outfile, True)
t1 = time.time()
augmentData(servers)
output("%.2f seconds to augment data." % (time.time() - t1), outfile, True)

displayResults(servers)
output("%.2f seconds total processing time." % (time.time() - ttotal), outfile, True)
output('Done!', outfile, True)
